{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pinterest API Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "### Pinterest API connection with paging\n",
    "def pinterest_paging(url = '', params = ''):\n",
    "    data = []\n",
    "    \n",
    "    response = requests.get(url = url, params = params)\n",
    "    response = response.json()\n",
    "    data.extend(response['data'])\n",
    "    \n",
    "    while response['page']['next'] != None:\n",
    "        response = requests.get(url = response['page']['next'])\n",
    "        response = response.json()\n",
    "        data.extend(response['data'])\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas.io.json import json_normalize\n",
    "\n",
    "access_token = 'YOUR_ACCESS_TOKEN'\n",
    "fields = 'id, board, note, counts'\n",
    "pin_data = pinterest_paging(url = 'https://api.pinterest.com/v1/me/pins/',\n",
    "params = {'access_token': access_token, 'fields': fields})\n",
    "pins = pd.io.json.json_normalize(pin_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pins.groupby('board.name')['board.name'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pinterest API Crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib2\n",
    "import time\n",
    "from selenium import webdriver\n",
    "\n",
    "import pprint\n",
    "\n",
    "reload(sys)  # Reload does the trick!\n",
    "sys.setdefaultencoding('UTF8')\n",
    "\n",
    "# Gets the specific filters available on the Pinterest page\n",
    "def get_subtopics(soup):\n",
    "    guideText = soup.findAll('span', {\"class\":\"guideText\"})\n",
    "\n",
    "    sub_topics = []\n",
    "    for guide in guideText:\n",
    "        sub_topics.append(str(guide.contents[0]))\n",
    "\n",
    "    print sub_topics\n",
    "\n",
    "#Gets the pin attributes for a given query\n",
    "def get_pins(soup,query_term):\n",
    "    #pins = soup.findAll('div', {\"class\":\"_ts _3m _tt\"})\n",
    "    pins = soup.findAll('div', {\"class\":\"pinWrapper\"})\n",
    "\n",
    "    all_pins = []\n",
    "    print(len(pins))\n",
    "    for pin in pins:\n",
    "        pinId = str(pin.find('div', {\"class\":\"pinHolder\"}).a['href'])\n",
    "        #re_pins =  str(pin.find('em', {\"class\":\"socialMetaCount repinCountSmall\"}).getText().strip())\n",
    "        #likes = str(pin.find('em', {\"class\":\"socialMetaCount likeCountSmall\"}).contents[0].strip())\n",
    "\n",
    "        pin_title = \"\"\n",
    "\n",
    "        if(pin.find('h3', {\"class\":\"richPinGridTitle\"})!=None):\n",
    "            pin_title = str(pin.find('h3', {\"class\":\"richPinGridTitle\"}).getText())\n",
    "\n",
    "        pin_description = \"\"\n",
    "        if (pin.find('p',{'class':'pinDescription'})!=None):\n",
    "            pin_description = str(pin.find('p',{'class':'pinDescription'}).getText().encode('utf-8').strip())\n",
    "\n",
    "        pin_user = \"\"\n",
    "        if (pin.find('div',{'class':'creditName'})!=None):\n",
    "            pin_user = str(pin.find('div',{'class':'creditName'}).getText().encode('utf-8').strip())\n",
    "\n",
    "        #print(pin_user)\n",
    "        pin_img = str(pin.find('div', {\"class\":\"pinHolder\"}).img['src'])\n",
    "        #break;\n",
    "\n",
    "        all_pins.append((pinId,pin_title.strip(),pin_description.strip(),pin_user,pin_img,query_term))\n",
    "\n",
    "    return(all_pins)\n",
    "\n",
    "        #print pinId + '\\t' + re_pins + '\\t' + '\\t' + pin_title + '\\t' #+ pin_description + '\\t' #+ pin_img+'\\t'+query_term\n",
    "\n",
    "def query_pinterest(query_term='forever21'):\n",
    "    \n",
    "    url = 'https://fr.pinterest.com/search/pins/?q='\n",
    "    query_term = query_term.replace(' ','+')\n",
    "    url = url + query_term\n",
    "    #print url\n",
    "    options = webdriver.ChromeOptions()\n",
    "    options.add_argument(\"--kiosk\")\n",
    "    driver = webdriver.Chrome(chrome_options=options,executable_path=\"/users/michal/py_workspace/book/pinterest_handlers/chromedriver\")\n",
    "\n",
    "    #login\n",
    "\n",
    "    driver.get(\"https://www.pinterest.com/login/\")\n",
    "    time.sleep(5)\n",
    "    username = driver.find_element_by_name(\"id\")\n",
    "    password = driver.find_element_by_name(\"password\")\n",
    "    username.send_keys(\"cn@thedatastrategy.com\")\n",
    "    password.send_keys(\"tds123,\")\n",
    "    login_attempt = driver.find_element_by_xpath(\"//*[@type='submit']\")\n",
    "    login_attempt.submit()\n",
    "\n",
    "    all_pins = []\n",
    "\n",
    "    time.sleep(10)\n",
    "    driver.get(url)\n",
    "    time.sleep(10)\n",
    "    for i in range(0,100):\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "        time.sleep(5)\n",
    "\n",
    "        html_page = driver.page_source\n",
    "\n",
    "        soup = BeautifulSoup(html_page)\n",
    "\n",
    "        all_pins = all_pins + get_pins(soup,query_term)\n",
    "        time.sleep(4)\n",
    "\n",
    "\n",
    "    #subtopics = get_subtopics(soup)\n",
    "    driver.close()\n",
    "    return(all_pins) #,subtopics)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "pins= query_pinterest('fashion')\n",
    "pins.columns = ['id','title','description','username','img','query_term']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bigram extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.collocations import *\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "def preprocess(text):\n",
    "\n",
    "    #1)Basic cleaning\n",
    "    text = text.strip()\n",
    "    text = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '',text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'[^\\w\\s]',' ',text)\n",
    "    text = text.lower()\n",
    "\n",
    "    #2) Tokenize single comment:\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "\n",
    "\n",
    "    #3) stopwords removal\n",
    "    stopwords_list = stopwords.words('english') + stopwords.words('french') + ['ici','les','mu','every','dautres','www','com']\n",
    "    tokens = [token for token in tokens if token not in stopwords_list]\n",
    "\n",
    "    return(tokens)\n",
    "\n",
    "\n",
    "pins['clean'] = pins['note'].apply(lambda x: preprocess(x))\n",
    "\n",
    "\n",
    "list_documents = pins['clean'].tolist()\n",
    "\n",
    "bigram_measures = nltk.collocations.BigramAssocMeasures()\n",
    "bigram_finder = BigramCollocationFinder.from_documents(list_documents)\n",
    "bigram_finder.apply_freq_filter(2)\n",
    "\n",
    "bigrams = bigram_finder.nbest(bigram_measures.raw_freq,20)\n",
    "freqs = bigram_finder.score_ngrams(bigram_measures.raw_freq)\n",
    "\n",
    "\n",
    "ngram = list(bigram_finder.ngram_fd.items())\n",
    "ngram.sort(key=lambda item: item[-1], reverse=True)\n",
    "\n",
    "frequency = [(\" \".join(k), v) for k,v in ngram]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build topic graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import networkx as nx\n",
    "\n",
    "columns = [t[0] for t in frequency]\n",
    "\n",
    "pins['clean_join'] = pins['clean'].apply(lambda x: \" \".join(x))\n",
    "\n",
    "for column in columns:\n",
    "    pins[column] = pins['clean_join'].str.lower().str.contains(column) #.astype(int)\n",
    "\n",
    "\n",
    "df_asint = pins[columns].astype(int)\n",
    "coocc = df_asint.T.dot(df_asint)\n",
    "coocc = coocc.values\n",
    "\n",
    "np.fill_diagonal(coocc, 0)\n",
    "\n",
    "G=nx.from_numpy_matrix(coocc)\n",
    "\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from networkx import node_connected_component\n",
    "\n",
    "connected = list(node_connected_component(G, (len(coocc)-1)))\n",
    "G = G.subgraph(connected)\n",
    "\n",
    "pos=nx.fruchterman_reingold_layout(G) # positions for all nodes\n",
    "\n",
    "# nodes\n",
    "nx.draw_networkx_nodes(G,pos,node_size=500,alpha=0.3,node_color='blue')\n",
    "\n",
    "# edges\n",
    "\n",
    "#undirected graph\n",
    "connected = list(node_connected_component(G, (len(coocc)-1)))\n",
    "G = G.subgraph(connected)\n",
    "\n",
    "nx.draw_networkx_edges(G,pos,width=2,alpha=0.2)\n",
    "\n",
    "labels = dict([(i,df_asint.columns[i]) for i in connected])\n",
    "nx.draw_networkx_labels(G,pos,labels,font_size=12)\n",
    "\n",
    "plt.axis('off')\n",
    "plt.savefig(\"topics_graph.png\") # save as png\n",
    "plt.show() # display\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build user graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.collocations import *\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import networkx as nx\n",
    "\n",
    "pins_scrap = pd.read_pickle(\"pins.pickle\")\n",
    "pins_scrap.columns = ['id','title','description','username','img','query_term']\n",
    "\n",
    "pins_scrap['title'] = pins_scrap['title'].apply(lambda x: x.encode('latin1').decode('utf8'))\n",
    "pins_scrap['description'] = pins_scrap['description'].apply(lambda x: x.encode('latin1').decode('utf8'))\n",
    "pins_scrap['username'] = pins_scrap['username'].apply(lambda x: x.encode('latin1').decode('utf8'))\n",
    "\n",
    "pins_scrap['title_desc'] = pins_scrap.apply(lambda x: x['title'] + \" \" + x['description'],axis=1)\n",
    "\n",
    "users = pins_scrap.groupby('username')['title_desc'].apply(lambda x: x.sum())\n",
    "users = pd.DataFrame(users)\n",
    "\n",
    "\n",
    "####Preprocessing\n",
    "\n",
    "def preprocess(text):\n",
    "\n",
    "    #1)Basic cleaning\n",
    "    text = text.strip()\n",
    "    text = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '',text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'[^\\w\\s]',' ',text)\n",
    "    text = text.lower()\n",
    "\n",
    "    #2) Tokenize single comment:\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "\n",
    "\n",
    "    #3) stopwords removal\n",
    "    stopwords_list = stopwords.words('english') + stopwords.words('french') + ['ici','les','mu','every','dautres','www','com','ã','tã','dã','faã','â','fr','co']\n",
    "    tokens = [token for token in tokens if token not in stopwords_list]\n",
    "\n",
    "    return(tokens)\n",
    "\n",
    "\n",
    "\n",
    "users['clean'] = users['title_desc'].apply(lambda x: preprocess(x))\n",
    "\n",
    "\n",
    "list_documents = users['clean'].tolist()\n",
    "\n",
    "bigram_measures = nltk.collocations.BigramAssocMeasures()\n",
    "bigram_finder = BigramCollocationFinder.from_documents(list_documents)\n",
    "bigram_finder.apply_freq_filter(2)\n",
    "\n",
    "bigrams = bigram_finder.nbest(bigram_measures.raw_freq,20)\n",
    "freqs = bigram_finder.score_ngrams(bigram_measures.raw_freq)\n",
    "\n",
    "\n",
    "ngram = list(bigram_finder.ngram_fd.items())\n",
    "ngram.sort(key=lambda item: item[-1], reverse=True)\n",
    "\n",
    "frequency = [(\" \".join(k), v) for k,v in ngram]\n",
    "#\n",
    "#\n",
    "#\n",
    "#####Extract features\n",
    "\n",
    "columns = [t[0] for t in frequency]\n",
    "\n",
    "#users = users.reset_index()\n",
    "usernames = users.index.tolist()\n",
    "\n",
    "users['clean_join'] = users['clean'].apply(lambda x: \" \".join(x))\n",
    "#\n",
    "for column in columns:\n",
    "    users[column] = users['clean_join'].str.lower().str.contains(column) #.astype(int)\n",
    "\n",
    "users = users[columns]\n",
    "users = users.transpose()\n",
    "\n",
    "df_asint = users[usernames].astype(int)\n",
    "coocc = df_asint.T.dot(df_asint)\n",
    "coocc = coocc.values\n",
    "\n",
    "np.fill_diagonal(coocc, 0)\n",
    "#\n",
    "\n",
    "\n",
    "#####graph\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from networkx import node_connected_component\n",
    "\n",
    "G=nx.from_numpy_matrix(coocc)\n",
    "\n",
    "connected = list(node_connected_component(G, (len(coocc)-1)))\n",
    "G = G.subgraph(connected)\n",
    "\n",
    "pos=nx.fruchterman_reingold_layout(G) # positions for all nodes\n",
    "\n",
    "# nodes\n",
    "nx.draw_networkx_nodes(G,pos,node_size=50,alpha=0.3,node_color='blue')\n",
    "\n",
    "# edges\n",
    "\n",
    "#undirected graph\n",
    "connected = list(node_connected_component(G, (len(coocc)-1)))\n",
    "G = G.subgraph(connected)\n",
    "\n",
    "nx.draw_networkx_edges(G,pos,width=1,alpha=0.2)\n",
    "\n",
    "labels = dict([(i,df_asint.columns[i]) for i in connected])\n",
    "nx.draw_networkx_labels(G,pos,labels,font_size=5)\n",
    "#\n",
    "# nx.draw_networkx_edges(G,pos,edgelist=esmall,width=6,alpha=0.5,edge_color='b',style='dashed')\n",
    "\n",
    "# labels\n",
    "#nx.draw_networkx_labels(G,pos,font_size=14,font_family='sans-serif')\n",
    "\n",
    "plt.axis('off')\n",
    "plt.savefig(\"users_graph.png\") # save as png\n",
    "plt.show() # display\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LIMIT = 25\n",
    "\n",
    "def centrality_measures(G,labels):\n",
    "\n",
    "    centralities = []\n",
    "    centralities.append(('degree centrality',nx.degree_centrality(G)))\n",
    "    centralities.append(('closeness centrality',nx.closeness_centrality(G)))\n",
    "    centralities.append(('betweenness centrality',nx.betweenness_centrality(G)))\n",
    "    centralities.append(('eigenvector centrality',nx.eigenvector_centrality(G)))\n",
    "\n",
    "    for centrality in centralities:\n",
    "        sorted_elements = sorted(centrality[1].items(), key=lambda x: x[1],reverse=True)\n",
    "\n",
    "        chart_labels = []\n",
    "        chart_measures = []\n",
    "        for element in sorted_elements[0:LIMIT]:\n",
    "            chart_labels.append(labels[element[0]])\n",
    "            chart_measures.append(element[1])\n",
    "\n",
    "        ax = plt.subplot()\n",
    "        ax.barh(range(0,len(chart_labels)), chart_measures, align='center', color='blue', ecolor='black')\n",
    "        ax.set_yticks(range(0,len(chart_labels)))\n",
    "        ax.set_yticklabels(chart_labels)\n",
    "        ax.invert_yaxis()  # labels read top-to-bottom\n",
    "        ax.set_xlabel(centrality[0])\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "def centrality_measures_heatmap(G,labels):\n",
    "\n",
    "    centralities = []\n",
    "    centralities.append(nx.degree_centrality(G))\n",
    "    centralities.append(nx.closeness_centrality(G))\n",
    "    centralities.append(nx.betweenness_centrality(G))\n",
    "    centralities.append(nx.eigenvector_centrality(G))\n",
    "\n",
    "    measures = []\n",
    "    for node in G.nodes_iter():\n",
    "        measures.append(list(map(lambda f: f[node], centralities)))\n",
    "        # print \"%s: %s\" % (node, measures)\n",
    "\n",
    "    measures = list(map(list, zip(*measures)))\n",
    "\n",
    "    #normalize\n",
    "    norm_measures = []\n",
    "    for measure in measures:\n",
    "        norm_measures.append([float(i)/max(measure) for i in measure])\n",
    "\n",
    "    column_labels = labels.values()\n",
    "    row_labels = ['degree','closeness','betweenness','eigenvector']\n",
    "    ax = plt.subplot()\n",
    "    ax.set_xticks(range(0,len(labels)))\n",
    "    ax.set_xticklabels(column_labels, minor=False)\n",
    "    ax.set_yticks([0,1,2,3])\n",
    "    ax.set_yticklabels(row_labels, minor=False)\n",
    "    plt.xticks(rotation=90)\n",
    "    plt.imshow(norm_measures, cmap='hot_r', interpolation='none')\n",
    "    plt.show()\n",
    "    #return(measures)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Community detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import community\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "#first compute the best partition\n",
    "partition = community.best_partition(G)\n",
    "\n",
    "#drawing\n",
    "size = float(len(set(partition.values())))\n",
    "pos = nx.spring_layout(G)\n",
    "count = 0.\n",
    "for community in set(partition.values()) :\n",
    "    count = count + 1.\n",
    "    list_nodes = [nodes for nodes in partition.keys() if partition[nodes] == community]\n",
    "    nx.draw_networkx_nodes(G, pos, list_nodes, node_size = 20,node_color = (count*random.uniform(0, 1)/size,count*random.uniform(0, 1)/size,count*random.uniform(0, 1)/size))\n",
    "\n",
    "nx.draw_networkx_edges(G,pos, alpha=0.1)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
